import facade
from parsel import Selector
from xjlibrary.our_file_dir import BaseDir

curpath = BaseDir.get_file_dir_absolute(__file__)
logspath = BaseDir.get_new_path(curpath, "logs")
rootDir = BaseDir.get_upper_dir(curpath, -2)
searchDir = BaseDir.get_new_path(rootDir, "download", "wanfag_cg", "download", "search")
BaseDir.create_dir(searchDir)
configfile = BaseDir.get_new_path(curpath, "db.ini")


class ParaSearch(object):
    def __init__(self):
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(configfile, "db", logger=self.logger)

    def parsing(self):
        self.allcount = 0
        for file in BaseDir.get_dir_all_files(searchDir):
            print(file)
            text = BaseDir.single_read_file(file)
            selector = Selector(text=text)
            divselect = selector.xpath('/html/body/div/div[@class="ResultList "]')
            values = []
            count = 0
            for div in divselect:
                longid = div.xpath('.//div[@class="ResultCont"]/div[@class="title"]/a[1]/@href').get().replace(
                    "/link.do?url=", "")
                standardid = div.xpath(
                    './/div[@class="ResultCont"]/div[@class="ResultMoreinfo"]/div[@class="author"]/span[@class="blockspan"]/text()').get()
                print((longid, standardid))
                # values.append((longid, standardid, file))
                sql = "insert into articlenew (longid,standardid) values ('%s','%s') on DUPLICATE key update longid='%s';"  # ;
                # values.append((longid, standardid,longid))
                sql = sql % (longid, standardid, longid)
                values.append(sql)
                count += 1
            print(count)
            self.allcount += count
            print(self.allcount)
            self.mysqlutils.ExeSqlListToDB(values)


if __name__ == "__main__":
    pars = ParaSearch()
    pars.parsing()
